import csv
import pandas as pd
import requests
from lxml import etree

#爬虫
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 '
                  'Safari/537.36',
    'Host': 'sz.lianjia.com',
    'Referer': 'https://sz.lianjia.com/ershoufang/'
}


def getUrl():
    area_list = ['yantianqu', 'luohuqu', 'futianqu', 'nanshanqu', 'baoanqu', 'longgangqu', 'longhuaqu', 'guangmingqu',
                 'pingshanqu', 'dapengxinqu']
    area_name = ['盐田区', '罗湖区', '福田区', '南山区', '宝安区', '龙岗区', '龙华区', '光明区', '坪山区', '大鹏新区']
    max1, min1 = 0, 0
    max2, min2 = 0, 0
    ytq, lhq, ftq, nsq, baq, lgq, lhq, gmq, psq, dpxq = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
    # first_url = 'https://sz.lianjia.com/ershoufang/pg1/'
    for j in area_name:
        for i in range(1, 101):
            url = 'https://sz.lianjia.com/ershoufang/%s/pg%s/' % (area_list[area_name.index(j)], str(i))
            getResponse(url, j)
            print('%s 第%d页数据获取完成！' % (j, i))
            if j == '盐田区':
                ytq += 1
            elif j == '罗湖区':
                lhq += 1
            elif j == '福田区':
                ftq += 1
            elif j == '南山区':
                nsq += 1
            elif j == '宝安区':
                baq += 1
            elif j == '龙岗区':
                lgq += 1
            elif j == '龙华区':
                lhq += 1
            elif j == '光明区':
                gmq += 1
            elif j == '坪山区':
                psq += 1
            elif j == '大鹏新区':
                dpxq += 1
    print(f'{ytq}, {lhq}, {ftq}, {nsq}, {baq}, {lgq}, {lhq}, {gmq}, {psq}, {dpxq}')
    # if getResponse(url, area_name) == -1:
    #     print(f'{j}数据获取完毕！')
    #     break


# 获取数据
def getResponse(url, area_name):
    res = requests.get(url, headers=headers)
    if res.status_code != 200:
        return
    else:
        res = res.text
        root = etree.HTML(res)
        length = len(root.xpath('//*[@id="content"]/div[1]/ul/li'))
        house_list = []
        for i in range(length):
            # 获取房屋的标题、位置、简介、总价、单价、跟踪信息等
            house_info = {"houseArea": area_name,
                          "houseTitle": root.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[1]/a/text()')[i],
                          "housePos": root.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[2]/div/a[1]/text()')[
                                          i] + '- ' + \
                                      root.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[2]/div/a[2]/text()')[i],
                          "houseInfo": root.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[3]/div/text()')[i],
                          "totalPrice": root.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[6]/div[1]/span/text()')[
                              i],
                          "unitPrice": root.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[6]/div[2]/span/text()')[
                              i],
                          "followInfo": root.xpath('//*[@id="content"]/div[1]/ul/li/div[1]/div[4]/text()')[i]}
            house_list.append(house_info)
        write_to_file(house_list)
    # return house_list


# 写入文件
def write_to_file(content):
    # ‘a’追加模式，‘utf_8_sig’格式到处csv不乱码
    with open('深圳二手房.csv', 'a', encoding='utf_8_sig', newline='') as f:
        fieldnames = ['houseTitle', 'houseArea', 'housePos', 'houseInfo', 'totalPrice', 'unitPrice', 'followInfo']
        # 利用csv包的DictWriter函数将字典格式数据存储到csv文件中
        for i in content:
            w = csv.DictWriter(f, fieldnames=fieldnames)
            w.writerow(i)


# 读取文件
def readfile():
    df = pd.read_csv("./深圳二手房.csv", encoding="utf-8")
    print(df.columns)
    print(df['totalPrice'])


if __name__ == '__main__':
    getUrl()
    # readfile()


#绘制饼状图
from pyecharts import options as opts
from pyecharts.charts import Pie
import pandas as pd

data_df = pd.read_csv('深圳二手房.csv')
def Pie_Base():
    # 商品售卖比列 火车
    v1 = ['盐田区', '罗湖区', '福田区', '南山区', '宝安区', '龙岗区', '龙华区', '光明区', '坪山区', '大鹏新区']
    # 深圳各区二手房数量 1265 5406 5598 5488 4206 11055 4089 486 958 343  总38894
    # 各区二手房占比深圳二手房比例
    v2 = [3.3, 14, 14.4, 14, 10.8, 28, 11, 1, 2.5, 1]

    c = (
        Pie()
        .add("", [list(z) for z in zip(v1, v2)])
        # 玫瑰图 --- 列表循环处理
        .set_global_opts(title_opts=opts.TitleOpts(title="深圳市二手房各区占比"), legend_opts=opts.LegendOpts(pos_left="80%"))
        .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}:{c}%"))

    )
    return c


Pie_Base().render("饼状图.html")



#绘制玫瑰图
from pyecharts.charts import Pie
from pyecharts import options as opts
import pandas as pd

data_df = pd.read_csv('深圳二手房.csv')
def Pie_RoseType():
    # 按照对应的年度和季度的数值 通过玫瑰图进行显示
    c = (
        Pie()
        .add("",
            [list(z) for z in zip(["201{}年/{}季度".format(y, z)
                                        for y in range(3)
                                        for z in range(1, 5)],
                                       [4.88, 5.88, 6.88, 7.88, 5.88, 7.88, 9.88, 8.88, 9.88, 5.88, 4.88, 6.88])],
                 # 内径和外径的设置
                 radius=["0%", "75%"],
                 rosetype="radius",
                 label_opts=opts.LabelOpts(is_show=True),
                 )
        .set_global_opts(title_opts=opts.TitleOpts(title="年季度玫瑰图显示"), legend_opts=opts.LegendOpts(pos_left="80%"))
    )
    return c


Pie_RoseType().render("玫瑰图.html")